from bs4 import BeautifulSoup
import sqlite3
import requests
import urllib.request
import time

def work():


    news_pool=["人民微看点","晚报","河南","上海","东北","广西","河北","山东","早报","肇庆","广东","共青团","陕西 ","每日经济新闻","新浪新闻综合","新京报","广东台触电新闻","人民","都市","日报","中国科技网","新浪","北京","天津","福建","厦门","南方","燕赵"]

    url_part="https://search.sina.com.cn/?q=%E6%8A%91%E9%83%81&c=news&from=&col=&range=all&source=&country=&size=10&stime=&etime=&time=&dpc=0&a=&ps=0&pf=0&page="
    num=0
    title=[]
    time_t=[]
    source=[]
    for i in range(20):
        try:
            num+=1
            url=url_part+str(num)
            r = requests.get(url)
            if (r.status_code == 200):
                html_content = urllib.request.urlopen(url).read()
                soup = BeautifulSoup(
                    html_content,
                    "html.parser"
                )
                # print(soup)

                temp=soup.find_all("div", class_="box-result clearfix")
                for i in temp:
                    tit=i.find("h2").find("a").get_text()
                    sour=i.find("h2").find("span").get_text().split(" ")[0]
                    ti=i.find("h2").find("span").get_text().split(" ")[1]
                    for j in news_pool:
                        if j in sour and tit not in title:
                            title.append(tit)
                            source.append(sour)
                            time_t.append(ti)
                            break
        except:
            print("error")
    conn = sqlite3.connect("./static/database/news.db")
    c = conn.cursor()
    try:
        c.execute("DROP TABLE NEWS")
        conn.commit()
        print("删除成功")
    except:
        print("创建成功")
    c.execute('''CREATE TABLE NEWS
                   (ID INT PRIMARY KEY     NOT NULL,
                   TITLE           TEXT    NOT NULL,
                   SOURCE          TEXT     NOT NULL,
                   TIME            TEXT);''')
    conn.commit()
    for i in range(len(title)):
        c.execute("INSERT INTO NEWS (ID,TITLE,SOURCE,TIME) VALUES ('"+str(i)+"', '"+title[i]+"', '"+source[i]+"', '"+time_t[i]+"')")

    conn.commit()




    conn.close()
    time.sleep(10)

while True:
    time_now = time.strftime("%H:%M:%S", time.localtime())  # 刷新
    if time_now == "24:00:00":
        print("开始爬虫")
        work()
        print("ok")

